#!/usr/bin/env julia

"""
    test_data_formats.jl

Comprehensive test suite for observation data format handling in GSI.
Tests BUFR/PrepBUFR decoding, data validation, and format conversion.

Test Coverage:
- BUFR message parsing and validation
- PrepBUFR data extraction and quality control flags
- Data format conversion between internal and external formats
- Error handling for malformed or corrupted data files
- Performance testing for large observation datasets
"""

using Test
using GSICoreAnalysis
using GSICoreAnalysis.ObservationProcessing.DataFormats
using Dates
using Random

"""Test BUFR message structure and parsing"""
@testset "BUFR Message Parsing" begin
    println("  Testing BUFR message parsing...")
    
    @testset "BUFR Header Validation" begin
        # Create mock BUFR header
        bufr_header = BUFRHeader(
            magic_number = "BUFR",
            total_length = 1024,
            edition_number = 4,
            originating_center = 7,    # NCEP
            originating_subcenter = 0,
            update_sequence = 0,
            optional_section = false,
            data_category = 12,        # Surface data
            local_subcategory = 255,
            master_table_version = 13,
            local_table_version = 1,
            year = 2024,
            month = 1,
            day = 1,
            hour = 12,
            minute = 0
        )
        
        @test bufr_header.magic_number == "BUFR"
        @test bufr_header.edition_number == 4
        @test bufr_header.originating_center == 7
        @test bufr_header.data_category == 12
        
        # Test header validation
        @test validate_bufr_header(bufr_header) == true
        
        # Test invalid header
        invalid_header = BUFRHeader(
            magic_number = "INVALID",
            total_length = 0,
            edition_number = 2,  # Unsupported edition
            originating_center = 999,
            data_category = 12
        )
        
        @test validate_bufr_header(invalid_header) == false
    end
    
    @testset "BUFR Data Descriptor Parsing" begin
        # Test standard WMO descriptors
        temp_descriptor = BUFRDescriptor(
            f_value = 0,    # Element descriptor
            x_value = 12,   # Temperature/humidity class
            y_value = 1     # Dry bulb temperature
        )
        
        @test temp_descriptor.f_value == 0
        @test temp_descriptor.x_value == 12
        @test temp_descriptor.y_value == 1
        @test get_descriptor_code(temp_descriptor) == "012001"
        
        # Test sequence descriptor
        seq_descriptor = BUFRDescriptor(
            f_value = 3,    # Sequence descriptor
            x_value = 1,    # Synoptic sequences
            y_value = 4     # Land station identification
        )
        
        @test seq_descriptor.f_value == 3
        @test get_descriptor_code(seq_descriptor) == "301004"
        
        # Test operator descriptor
        op_descriptor = BUFRDescriptor(
            f_value = 2,    # Operator descriptor
            x_value = 1,    # Change data width
            y_value = 0     # Reset to original width
        )
        
        @test op_descriptor.f_value == 2
        @test get_descriptor_code(op_descriptor) == "201000"
    end
    
    @testset "BUFR Data Extraction" begin
        # Create mock BUFR message with surface observations
        bufr_msg = BUFRMessage{Float64}()
        
        # Add station identification
        add_bufr_element!(bufr_msg, "001001", 47681)     # WMO station ID
        add_bufr_element!(bufr_msg, "001002", 1)         # WMO station type
        add_bufr_element!(bufr_msg, "005001", 40.78)     # Latitude
        add_bufr_element!(bufr_msg, "006001", -73.97)    # Longitude  
        add_bufr_element!(bufr_msg, "007001", 10.0)      # Height of station
        
        # Add observation time
        add_bufr_element!(bufr_msg, "004001", 2024)      # Year
        add_bufr_element!(bufr_msg, "004002", 1)         # Month
        add_bufr_element!(bufr_msg, "004003", 1)         # Day
        add_bufr_element!(bufr_msg, "004004", 12)        # Hour
        add_bufr_element!(bufr_msg, "004005", 0)         # Minute
        
        # Add meteorological data
        add_bufr_element!(bufr_msg, "010004", 101325.0)  # Pressure (Pa)
        add_bufr_element!(bufr_msg, "012001", 288.15)    # Temperature (K)
        add_bufr_element!(bufr_msg, "013003", 75.0)      # Relative humidity (%)
        add_bufr_element!(bufr_msg, "011001", 5.0)       # Wind direction (degrees)
        add_bufr_element!(bufr_msg, "011002", 8.0)       # Wind speed (m/s)
        
        # Test data extraction
        station_id = get_bufr_element(bufr_msg, "001001")
        @test station_id == 47681
        
        latitude = get_bufr_element(bufr_msg, "005001")
        @test latitude ≈ 40.78
        
        pressure = get_bufr_element(bufr_msg, "010004")
        @test pressure ≈ 101325.0
        
        temperature = get_bufr_element(bufr_msg, "012001")
        @test temperature ≈ 288.15
        
        # Test missing data handling
        missing_element = get_bufr_element(bufr_msg, "999999")
        @test ismissing(missing_element)
    end
end

"""Test PrepBUFR data processing"""
@testset "PrepBUFR Processing" begin
    println("  Testing PrepBUFR processing...")
    
    @testset "PrepBUFR Report Structure" begin
        # Create mock PrepBUFR report
        prepbufr_report = PrepBUFRReport{Float64}(
            station_id = "KORD",
            station_type = 181,  # Surface land station
            latitude = 41.995,
            longitude = -87.933,
            elevation = 201.0,
            observation_time = DateTime(2024, 1, 1, 12, 0, 0)
        )
        
        @test prepbufr_report.station_id == "KORD"
        @test prepbufr_report.station_type == 181
        @test prepbufr_report.latitude ≈ 41.995
        @test prepbufr_report.longitude ≈ -87.933
        
        # Add observation data
        add_prepbufr_observation!(prepbufr_report, PrepBUFRObservation{Float64}(
            variable_type = "PS",      # Surface pressure
            pressure_level = missing,  # Surface level
            observed_value = 1013.25,
            observation_error = 1.0,
            background_value = 1015.0,
            analysis_value = 1013.8,
            quality_mark = 2,          # Good
            program_code = 1,
            observation_type = 181
        ))
        
        add_prepbufr_observation!(prepbufr_report, PrepBUFRObservation{Float64}(
            variable_type = "T",       # Temperature
            pressure_level = missing,  # Surface
            observed_value = 285.5,
            observation_error = 1.2,
            background_value = 286.2,
            analysis_value = 285.7,
            quality_mark = 2,          # Good
            program_code = 1,
            observation_type = 181
        ))
        
        @test length(prepbufr_report.observations) == 2
        
        # Test observation retrieval by type
        ps_obs = get_observations_by_type(prepbufr_report, "PS")
        @test length(ps_obs) == 1
        @test ps_obs[1].observed_value ≈ 1013.25
        
        temp_obs = get_observations_by_type(prepbufr_report, "T")
        @test length(temp_obs) == 1
        @test temp_obs[1].observed_value ≈ 285.5
    end
    
    @testset "PrepBUFR Quality Control Flags" begin
        # Create observation with various QC flags
        qc_obs = PrepBUFRObservation{Float64}(
            variable_type = "T",
            observed_value = 290.0,
            observation_error = 1.5,
            background_value = 288.0,
            quality_mark = 3,          # Suspect
            program_code = 1,
            observation_type = 120     # Radiosonde
        )
        
        @test qc_obs.quality_mark == 3
        @test is_observation_suspect(qc_obs) == true
        @test is_observation_good(qc_obs) == false
        @test is_observation_rejected(qc_obs) == false
        
        # Test rejected observation
        rejected_obs = PrepBUFRObservation{Float64}(
            variable_type = "UV",      # Wind
            observed_value = 25.0,
            observation_error = 2.0,
            background_value = 8.0,
            quality_mark = 15,         # Rejected
            program_code = 1,
            observation_type = 220
        )
        
        @test is_observation_rejected(rejected_obs) == true
        @test is_observation_good(rejected_obs) == false
        
        # Test good observation
        good_obs = PrepBUFRObservation{Float64}(
            variable_type = "Q",       # Specific humidity
            observed_value = 0.008,
            observation_error = 0.001,
            background_value = 0.0082,
            quality_mark = 2,          # Good
            program_code = 1,
            observation_type = 120
        )
        
        @test is_observation_good(good_obs) == true
        @test is_observation_suspect(good_obs) == false
        @test is_observation_rejected(good_obs) == false
    end
    
    @testset "PrepBUFR Data Conversion" begin
        # Create PrepBUFR report
        report = PrepBUFRReport{Float64}(
            station_id = "72202",
            station_type = 120,        # Radiosonde
            latitude = 40.0,
            longitude = -75.0,
            elevation = 50.0,
            observation_time = DateTime(2024, 1, 1, 12, 0, 0)
        )
        
        # Add multi-level temperature observations
        pressure_levels = [1000.0, 850.0, 700.0, 500.0, 300.0, 200.0]
        temperatures = [288.0, 283.0, 278.0, 265.0, 245.0, 220.0]
        
        for (p, t) in zip(pressure_levels, temperatures)
            add_prepbufr_observation!(report, PrepBUFRObservation{Float64}(
                variable_type = "T",
                pressure_level = p,
                observed_value = t,
                observation_error = 1.0,
                background_value = t + randn() * 0.5,
                analysis_value = t + randn() * 0.3,
                quality_mark = 2,
                program_code = 1,
                observation_type = 120
            ))
        end
        
        # Convert to GSI observation format
        gsi_observations = convert_prepbufr_to_gsi(report)
        
        @test length(gsi_observations) == length(pressure_levels)
        
        for (i, obs) in enumerate(gsi_observations)
            @test obs isa ConventionalObservation{Float64}
            @test obs.obs_type == "temperature"
            @test obs.latitude == report.latitude
            @test obs.longitude == report.longitude
            @test obs.pressure ≈ pressure_levels[i]
            @test obs.value ≈ temperatures[i]
        end
    end
end

"""Test data format validation and error handling"""
@testset "Data Format Validation" begin
    println("  Testing data format validation...")
    
    @testset "BUFR Data Validation" begin
        # Test valid data ranges
        valid_temp = 273.15
        @test validate_temperature(valid_temp) == true
        
        # Test invalid temperature
        invalid_temp = 500.0  # Too hot
        @test validate_temperature(invalid_temp) == false
        
        cold_temp = 100.0     # Too cold
        @test validate_temperature(cold_temp) == false
        
        # Test pressure validation
        valid_pressure = 101325.0
        @test validate_pressure(valid_pressure) == true
        
        invalid_pressure = 50000.0  # Too low for surface
        @test validate_pressure(invalid_pressure) == false
        
        high_pressure = 110000.0    # Too high
        @test validate_pressure(high_pressure) == false
        
        # Test coordinate validation
        @test validate_latitude(45.0) == true
        @test validate_latitude(-89.5) == true
        @test validate_latitude(91.0) == false   # Invalid
        @test validate_latitude(-95.0) == false  # Invalid
        
        @test validate_longitude(180.0) == true
        @test validate_longitude(-180.0) == true
        @test validate_longitude(181.0) == false   # Invalid
        @test validate_longitude(-185.0) == false  # Invalid
    end
    
    @testset "Missing Data Handling" begin
        # Create BUFR message with missing data
        bufr_msg = BUFRMessage{Float64}()
        
        # Add station info
        add_bufr_element!(bufr_msg, "001001", 12345)
        add_bufr_element!(bufr_msg, "005001", 40.0)
        add_bufr_element!(bufr_msg, "006001", -100.0)
        
        # Add missing temperature (using BUFR missing value)
        add_bufr_element!(bufr_msg, "012001", BUFR_MISSING_FLOAT64)
        
        # Add valid pressure
        add_bufr_element!(bufr_msg, "010004", 101325.0)
        
        # Test missing data detection
        temp = get_bufr_element(bufr_msg, "012001")
        @test ismissing(temp) || is_bufr_missing(temp)
        
        pressure = get_bufr_element(bufr_msg, "010004")
        @test !ismissing(pressure)
        @test !is_bufr_missing(pressure)
        
        # Test conversion with missing data
        obs_list = convert_bufr_to_observations(bufr_msg)
        
        # Should only create pressure observation, not temperature
        pressure_obs = filter(obs -> obs.obs_type == "surface_pressure", obs_list)
        temp_obs = filter(obs -> obs.obs_type == "temperature", obs_list)
        
        @test length(pressure_obs) == 1
        @test length(temp_obs) == 0  # Missing temperature should be skipped
    end
    
    @testset "Data Quality Assessment" begin
        # Create dataset with various quality issues
        bufr_messages = BUFRMessage{Float64}[]
        
        # Good quality message
        good_msg = BUFRMessage{Float64}()
        add_bufr_element!(good_msg, "001001", 12345)
        add_bufr_element!(good_msg, "005001", 40.0)
        add_bufr_element!(good_msg, "006001", -100.0)
        add_bufr_element!(good_msg, "012001", 288.15)
        add_bufr_element!(good_msg, "010004", 101325.0)
        push!(bufr_messages, good_msg)
        
        # Message with suspect temperature
        suspect_msg = BUFRMessage{Float64}()
        add_bufr_element!(suspect_msg, "001001", 12346)
        add_bufr_element!(suspect_msg, "005001", 41.0)
        add_bufr_element!(suspect_msg, "006001", -101.0)
        add_bufr_element!(suspect_msg, "012001", 350.0)  # Suspect temperature
        add_bufr_element!(suspect_msg, "010004", 101000.0)
        push!(bufr_messages, suspect_msg)
        
        # Message with invalid coordinates
        invalid_msg = BUFRMessage{Float64}()
        add_bufr_element!(invalid_msg, "001001", 12347)
        add_bufr_element!(invalid_msg, "005001", 95.0)   # Invalid latitude
        add_bufr_element!(invalid_msg, "006001", -100.0)
        add_bufr_element!(invalid_msg, "012001", 285.0)
        push!(bufr_messages, invalid_msg)
        
        # Assess data quality
        quality_report = assess_bufr_quality(bufr_messages)
        
        @test quality_report.total_messages == 3
        @test quality_report.valid_messages == 1
        @test quality_report.suspect_messages == 1
        @test quality_report.invalid_messages == 1
        
        # Test detailed quality metrics
        @test haskey(quality_report.issues, "invalid_coordinates")
        @test haskey(quality_report.issues, "suspect_temperature")
    end
end

"""Test file I/O operations"""
@testset "File I/O Operations" begin
    println("  Testing file I/O operations...")
    
    @testset "BUFR File Reading" begin
        # Create mock BUFR file data
        mock_bufr_data = create_mock_bufr_file()
        
        # Test file format detection
        @test detect_file_format(mock_bufr_data) == :bufr
        
        # Test BUFR file parsing
        bufr_messages = parse_bufr_file(mock_bufr_data)
        @test length(bufr_messages) > 0
        
        # Validate parsed messages
        for msg in bufr_messages
            @test validate_bufr_message(msg) == true
        end
    end
    
    @testset "PrepBUFR File Reading" begin
        # Create mock PrepBUFR file
        mock_prepbufr_data = create_mock_prepbufr_file()
        
        # Test file format detection
        @test detect_file_format(mock_prepbufr_data) == :prepbufr
        
        # Test PrepBUFR parsing
        prepbufr_reports = parse_prepbufr_file(mock_prepbufr_data)
        @test length(prepbufr_reports) > 0
        
        # Validate parsed reports
        for report in prepbufr_reports
            @test validate_prepbufr_report(report) == true
        end
    end
    
    @testset "Large File Processing" begin
        # Test processing of large observation files
        n_messages = 1000
        large_bufr_data = create_large_mock_bufr_file(n_messages)
        
        # Test streaming parser for memory efficiency
        message_count = 0
        total_observations = 0
        
        process_bufr_file_streaming(large_bufr_data) do msg
            message_count += 1
            obs_list = convert_bufr_to_observations(msg)
            total_observations += length(obs_list)
        end
        
        @test message_count == n_messages
        @test total_observations > 0
        
        # Test that memory usage remains reasonable
        # (In practice, this would check actual memory usage)
    end
end

"""Test performance characteristics"""
@testset "Performance Testing" begin
    println("  Testing performance characteristics...")
    
    @testset "Parsing Performance" begin
        # Create dataset for performance testing
        n_messages = 100
        bufr_data = create_large_mock_bufr_file(n_messages)
        
        # Time the parsing operation
        start_time = time()
        messages = parse_bufr_file(bufr_data)
        parse_time = time() - start_time
        
        @test length(messages) == n_messages
        @test parse_time < 10.0  # Should parse 100 messages in under 10 seconds
        
        # Test conversion performance
        start_time = time()
        all_observations = ConventionalObservation{Float64}[]
        for msg in messages
            obs_list = convert_bufr_to_observations(msg)
            append!(all_observations, obs_list)
        end
        conversion_time = time() - start_time
        
        @test length(all_observations) > 0
        @test conversion_time < 5.0  # Conversion should be fast
    end
    
    @testset "Memory Efficiency" begin
        # Test that large datasets don't cause memory issues
        # This would ideally use actual memory profiling tools
        
        n_large = 500
        large_dataset = create_large_mock_bufr_file(n_large)
        
        # Process in chunks to test memory management
        chunk_size = 50
        total_processed = 0
        
        for i in 1:chunk_size:n_large
            chunk_end = min(i + chunk_size - 1, n_large)
            chunk_data = extract_bufr_chunk(large_dataset, i, chunk_end)
            
            messages = parse_bufr_file(chunk_data)
            total_processed += length(messages)
            
            # Force garbage collection to test memory cleanup
            GC.gc()
        end
        
        @test total_processed == n_large
    end
end

"""Helper functions for creating mock data"""

function create_mock_bufr_file()
    # Create a small mock BUFR file with valid structure
    bufr_data = UInt8[]
    
    # Add BUFR magic number
    append!(bufr_data, Vector{UInt8}("BUFR"))
    
    # Add mock message length (24 bytes for minimal message)
    append!(bufr_data, reinterpret(UInt8, [UInt32(24)]))
    
    # Add edition number (4)
    push!(bufr_data, 0x04)
    
    # Add minimal valid BUFR structure
    # This is highly simplified - real BUFR files are more complex
    append!(bufr_data, zeros(UInt8, 15))  # Padding to make 24 bytes total
    
    return bufr_data
end

function create_mock_prepbufr_file()
    # Create mock PrepBUFR file data
    # PrepBUFR files have their own binary format
    prepbufr_data = UInt8[]
    
    # Add PrepBUFR identifier
    append!(prepbufr_data, Vector{UInt8}("PREP"))
    
    # Add minimal structure for testing
    append!(prepbufr_data, zeros(UInt8, 100))
    
    return prepbufr_data
end

function create_large_mock_bufr_file(n_messages::Int)
    # Create a large mock BUFR file for performance testing
    bufr_data = UInt8[]
    
    for i in 1:n_messages
        # Add each message
        msg_data = create_mock_bufr_file()
        append!(bufr_data, msg_data)
    end
    
    return bufr_data
end

println("  ✓ All data format tests completed successfully")